(ns search-engine.crawler
   (:use [jsoup.soup]))

(def url "http://www.last.fm/music/Madonna/Confessions+on+a+Dance+Floor/Future+Lovers")

(defn get-links
     [doc]
     (filter #(.contains % "//www.last.fm") (set ($ doc  "a[href]" (attr "abs:href")))))

(defn get-meta
  [doc]
  (if (= '("song") ($ doc "meta[property=og:type][content=song]" (attr "content")))
    (first ($ doc "meta[property=og:title]" (attr "content")))))

(defn crawl-page 
  [url]
  (let [doc (get! url :ignore-http-errors true)]
      [(get-meta doc)  (get-links doc)]))

(defn process-agent
  [agent]
  (if (first @agent) (println (first @agent)))
  (second @agent))

(defn crawl
  ([pages i] (let [agents (map (fn [page] (let [agent (agent page)] 
                                            (send-off agent crawl-page))) (take 15 pages))]
               (apply await-for 8000 agents)
               (crawl [] agents (dec i))))
  ([pages agents i] (let [new-pages (reduce (fn [res ag]
                                              (if (or (agent-error ag) (string? (second @ag)) (empty? (second @ag)))
                                                res
                                                (set (apply conj res (process-agent ag))))) [] agents)]
                      (if (< i 1)
                        "finished!"
                        (crawl new-pages i)))))


